﻿

namespace Trader.Crawling
{
    using System;
    using System.Collections.Generic;
    using System.Linq;
    using System.Text;
    using System.IO;
    using System.Net;
    using System.Text.RegularExpressions;
    using System.Xml.XPath;
    using Trader.DataServices.Interface;
    using Trader.BO.Interface;

    public class Crawl
    {
        IDataProvider data;
        //Dictionary<IStock, Dictionary<DateTime,int> > stockSearchResults;

        public Crawl(IDataProvider d)
        {
            data = d;
        }

        public void webConnect()
        {
            try
            {
                // Download the data to a buffer.
                
                DateTime date = new DateTime(2000, 1, 1);

                foreach (IStock stock in data.Stocks)
                {
                    
                    string currStock = stock.Name.Replace('/', ' ');

                    // Create a file for each stock containing number of google results in each week
                    StreamWriter googleResults = File.CreateText(string.Format("c:\\StockData\\{0}.txt", currStock));

                    currStock = currStock.Replace('-', ' ');
                    currStock = currStock.Replace('&', ' ');
                    currStock = currStock.Replace('/', ' ');
                    currStock = currStock.Replace(' ', '+');

                    System.Threading.ManualResetEvent[] waits = new System.Threading.ManualResetEvent[800];
                    int i = 0;
                    // Two threads are alowed because of google limitation
                    System.Threading.ThreadPool.SetMaxThreads(2, 2);

                    for (DateTime d = new DateTime(2000, 1, 1);
                            d <= new DateTime(2008, 1, 1) && i < 800;
                            d += TimeSpan.FromDays(7), i++)
                    {
                        int j = i;
                        DateTime currDate = d;
                        waits[i] = new System.Threading.ManualResetEvent(false);
                        System.Threading.ThreadPool.QueueUserWorkItem(
                    delegate
                    {
                        try
                        {
                            WebClient client = new WebClient();
                            // Do the search
                            String pageDataString = 
                                client.DownloadString(
                                    string.Format("http://www.google.com/search?hl=en&q={0}+{1}&btnG=%D7%97%D7%99%D7%A4%D7%95%D7%A9&lr=",
                                    currStock, currDate.ToString("dd-MM-yyyy")));

                            // Parse the page to get the results number
                            Regex r = new Regex(@"Results <b>1</b> - <b>10</b> of about <b>[\d,]+</b>");
                            string s = r.Match(pageDataString).Value;
                            s = s.Remove(0, "Results <b>1</b> - <b>10</b> of about <b>".Length);
                            s = s.Remove(s.Length - 4).Replace(",", string.Empty);
                            int num = int.Parse(s);

                            // Write it to the file with locking mechanism
                            lock (googleResults)
                            {
                                googleResults.WriteLine("{0}   {1}", currDate.ToString("dd-MM-yyyy"), num.ToString());
                            }

                            waits[j].Set();
                        }
                        catch (Exception e)
                        {
                            waits[j].Set();
                        }
                    });

                    }
                    for (int k = 0; k < i; k++)
                    {
                        waits[k].WaitOne();
                    }

                    googleResults.Close();  
                } // End Foreach
            }
            catch (Exception e)
            {
            }
        }   
    }
}
